{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.distributions import Normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Use CUDA</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "device   = torch.device(\"cuda\" if use_cuda else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Create Environments</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from common.multiprocessing_env import SubprocVecEnv\n",
    "\n",
    "num_envs = 16\n",
    "env_name = \"Pendulum-v0\"\n",
    "\n",
    "def make_env():\n",
    "    def _thunk():\n",
    "        env = gym.make(env_name)\n",
    "        return env\n",
    "\n",
    "    return _thunk\n",
    "\n",
    "envs = [make_env() for i in range(num_envs)]\n",
    "envs = SubprocVecEnv(envs)\n",
    "\n",
    "env = gym.make(env_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Neural Network</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_weights(m):\n",
    "    if isinstance(m, nn.Linear):\n",
    "        nn.init.normal_(m.weight, mean=0., std=0.1)\n",
    "        nn.init.constant_(m.bias, 0.1)\n",
    "        \n",
    "\n",
    "class ActorCritic(nn.Module):\n",
    "    def __init__(self, num_inputs, num_outputs, hidden_size, std=0.0):\n",
    "        super(ActorCritic, self).__init__()\n",
    "        \n",
    "        self.critic = nn.Sequential(\n",
    "            nn.Linear(num_inputs, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, 1)\n",
    "        )\n",
    "        \n",
    "        self.actor = nn.Sequential(\n",
    "            nn.Linear(num_inputs, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, num_outputs),\n",
    "        )\n",
    "        self.log_std = nn.Parameter(torch.ones(1, num_outputs) * std)\n",
    "        \n",
    "        self.apply(init_weights)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        value = self.critic(x)\n",
    "        mu    = self.actor(x)\n",
    "        std   = self.log_std.exp().expand_as(mu)\n",
    "        dist  = Normal(mu, std)\n",
    "        return dist, value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(frame_idx, rewards):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, rewards[-1]))\n",
    "    plt.plot(rewards)\n",
    "    plt.show()\n",
    "    \n",
    "def test_env(vis=False):\n",
    "    state = env.reset()\n",
    "    if vis: env.render()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "    while not done:\n",
    "        state = torch.FloatTensor(state).unsqueeze(0).to(device)\n",
    "        dist, _ = model(state)\n",
    "        next_state, reward, done, _ = env.step(dist.sample().cpu().numpy()[0])\n",
    "        state = next_state\n",
    "        if vis: env.render()\n",
    "        total_reward += reward\n",
    "    return total_reward"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>GAE</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_gae(next_value, rewards, masks, values, gamma=0.99, tau=0.95):\n",
    "    values = values + [next_value]\n",
    "    gae = 0\n",
    "    returns = []\n",
    "    for step in reversed(range(len(rewards))):\n",
    "        delta = rewards[step] + gamma * values[step + 1] * masks[step] - values[step]\n",
    "        gae = delta + gamma * tau * masks[step] * gae\n",
    "        returns.insert(0, gae + values[step])\n",
    "    return returns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> Proximal Policy Optimization Algorithm</h1>\n",
    "<h2><a href=\"https://arxiv.org/abs/1707.06347\">Arxiv</a></h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ppo_iter(mini_batch_size, states, actions, log_probs, returns, advantage):\n",
    "    batch_size = states.size(0)\n",
    "    for _ in range(batch_size // mini_batch_size):\n",
    "        rand_ids = np.random.randint(0, batch_size, mini_batch_size)\n",
    "        yield states[rand_ids, :], actions[rand_ids, :], log_probs[rand_ids, :], returns[rand_ids, :], advantage[rand_ids, :]\n",
    "        \n",
    "        \n",
    "\n",
    "def ppo_update(ppo_epochs, mini_batch_size, states, actions, log_probs, returns, advantages, clip_param=0.2):\n",
    "    for _ in range(ppo_epochs):\n",
    "        for state, action, old_log_probs, return_, advantage in ppo_iter(mini_batch_size, states, actions, log_probs, returns, advantages):\n",
    "            dist, value = model(state)\n",
    "            entropy = dist.entropy().mean()\n",
    "            new_log_probs = dist.log_prob(action)\n",
    "\n",
    "            ratio = (new_log_probs - old_log_probs).exp()\n",
    "            surr1 = ratio * advantage\n",
    "            surr2 = torch.clamp(ratio, 1.0 - clip_param, 1.0 + clip_param) * advantage\n",
    "\n",
    "            actor_loss  = - torch.min(surr1, surr2).mean()\n",
    "            critic_loss = (return_ - value).pow(2).mean()\n",
    "\n",
    "            loss = 0.5 * critic_loss + actor_loss - 0.001 * entropy\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_inputs  = envs.observation_space.shape[0]\n",
    "num_outputs = envs.action_space.shape[0]\n",
    "\n",
    "#Hyper params:\n",
    "hidden_size      = 256\n",
    "lr               = 3e-4\n",
    "num_steps        = 20\n",
    "mini_batch_size  = 5\n",
    "ppo_epochs       = 4\n",
    "threshold_reward = -200\n",
    "\n",
    "model = ActorCritic(num_inputs, num_outputs, hidden_size).to(device)\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_frames = 15000\n",
    "frame_idx  = 0\n",
    "test_rewards = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAE/CAYAAABLrsQiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8FPX9x/HXJydXCPd933IIQrjqLYpCtdrWA2+rglatte1Pq/X3a2vv22ptVVS8BTxbFJBi1VqrnIKEGwTkCoQzhCshyef3xwy6Ro5Aspkk+34+Hnlkd75zfHZ2d94735ndMXdHREQSV1LUBYiISLQUBCIiCU5BICKS4BQEIiIJTkEgIpLgFAQiIglOQVDBzKy7mc03s3wzuz3qeqT8zGyNmZ0ddR0i8aIgqHh3Ae+4e4a7Pxh1MaWZ2QVmttDMdpvZB2bWM6bNzOwXZrbBzPLM7F0z6xXT3sjMJprZNjPbambPm1n9mPYOZvaOme01s6XaeIKZpZnZy2GYuJmdUao93cweMbPNZrbdzF43s9Yx7R3MbIqZ7TCzTWb2kJmlHGZZLc1skpltDJfVoVR7azP7R7ic9WZ2c6l2N7M94Wtjt5k9HtP2UzM7ENO228w6xbSfZWYfmdkuM1tlZmNKzbupmb0Qvq52mNnzpdrPDqffE9Z2aUzbWDNbZmYlZnZdqemuNbO54XLXm9nvYtdP+Jp9LZzvp2Z2RUzbj0o9nn3hMpqE7X8wsxXhh7qlZnZNzLRNzOy/4Xthp5l9aGYnH+p5qRbcXX8V+Ae8Bdx4hPbkCGvrCuwCTgFSgHuAlUBK2H4psBHoBCQDvwY+ipn+b8A/gfpAZvhY/xTT/iHwJ6A28E1gJ9D0OOpMiWj9HHK5wBrg7OOcZxpwR7jOc4AzSrXfBXwMNAdqAc8Ar8a0TwGeCttaANnA7YdZVnPgFmAo4ECHUu3vAH8GUoG+wHbgzJh2B7ocZt4/BZ47TFsqkAfcBBgwENgN9I0Z5z/hayMzHP+kmLaeQC4wInxdNgY6x7TfCgwD5gDXlVr2t4FTw/XcGpgL3B3TPh6YCNQLn4M8oNcRHuPbMffvA3oQfGAeDOwAvhK21QK6h20GXBSuz0heu+V+7UddQE36A94GioH94RuhW/gmfjh8Q+8Bzga+Cswj2CivA34aM48O4RvyW2HbDuDm8M21gGDj+lCp5V4PLAnHnQa0P0x9twGTY+4nAfuAYeH9HwIvxrT3AvbH3J8K3BJz/1ZgWni7G1AAZMS0/we4uYzrbk24/AXhfFKAVsArwBZgNeEGMHwT7gOahPfvBYqA+uH9nwN/Dm+XZV3fAKwF3guHXw18CmwL572G4wyCUo9xPV8OgoeB38Xc/yqwLOb+EmBkzP3fA48eZTkplAoCgg2hExPMwFjg2Zj7xxsEzcNp68QMmw1cHt4eHq7DQ34IAl4Afl6G9fc+pYLgEON8H3g9vF0XKAS6xbQ/C/zmENMZsAq49gjzngT84BDDk4ALwnXQrLyvkyj+1DVUgdz9LIKN323uXs/dl4dNVwC/BDIIXsx7gGuABgRv/G+b2UWlZjeY4BP8ZQSf4u4lCJFewKVmdjqAmV0I/Aj4BtA0XP74I5RppW4b0Du8PwHobGbdzCwVuBZ4M2b8vwLnm1lDM2tI8Kl/atjWC1jl7vkx438cDi+rywnWRwOgBHg9nEdrgk+Ed5jZue6+n2BDc3o43ekEG+6TY+7/O7xdlnV9OnACcG7YVfYwQRi0Ivh02ubgiGZ2ipntPIbHdDRPACebWSszqwNcyefrFILnfpSZ1Qm7jEbwxeekrKzU/4O3e5ca772wC+rV0l1LwAVht9IiM/v2wYHuvpngNfctM0s2s6FAe4LXOsAQYBnwdNiVMvvg6zemHTPLNrMcM3vOzBodx2MEOA1YFN7uBhTFvA/h8K/JU4FmBB88vsTMahN8GFtUavgCgg9+k4DH3T33OOuOVtRJVNP+gHeJ6Roi2CN45ijT/Bm4P7zdgeCTReuY9m3AZTH3XwHuCG9PBW6IaUsC9nKIvQKC3dw9wBkEu9L/R7DBvSdsTwMeCJdfRPApvGPM9K0IuoNKwr/pQFrYdjUwo9Tyfgk8Vcb1tga4Pub+YGBtqXHuAZ4Mb/8ceJDg0+8m4LvAb/h8b6HxMazrTjHtPwYmxNw/+KkyXnsEmQQBfHCdzwMaxbSfQNDdURSO8xRgR1nOl/YIwuHvA38J11F/gq6M2L2P08LXQAPgIWAhn3cb9gyf/2TgKwTdXJfHTHsBsDmsswgYHdM2ls/3vFKBUQR7tgf36ArD578bwZ7LK8Dzh3hcR9wjINgzXh8z31OBTaXGGQ28e4hpnzjSaxV4miCAv7Tuw/V5OUfYm6jqf9ojqBzrYu+Y2eDwoOoWM8sj6PppUmqazTG39x3ifr3wdnvggfCA1U6CN7cRfIr+AndfSvAp/yGCN3ITYDHBmweCjeBAoC3Bi/s+4O3wkyrAi8Bygj2b+sAnwHNh2+5wWKz6QD5lF7ue2gOtDj6u8LH9iKAbAoJP/GcQbNCyCULpdIJPlyvdfRuUeV3HLrdV7H1330MQxEdlZu1iDz6W6REHe1npBHsedYFXCfcIzCyJYOPzatjWBGgI/LaM8y7tSqAjweN7mOC5O/jc4+7vuXuhu+8kCNaOBEGEuy92943uXuzuHxB8YLg4rLMHQZhdQxAkvYC7zOyr4az3AWvc/Ql3P+DuE8IaTo5pf9Ldl7v7buBXwMhjeWDhXt6vgRHuvjUcXKbXZPj6voRgY3+oef+eYM/pUg+3/LHcfb+7jwfuNrO+x1J3VaEgqBylXzwvEOxKtnX3TOARvrjLfizWATe5e4OYv9rhm/XLhbi/7O693b0x8BOCT8Wzw+Z+wER3X+/uRe7+FMGGp2dM+6Puvid8wz7C52/YRUAnM8uIWVxfSu1KH0XseloHrC71uDLc/eDyPiA4WPd14N/uvhhoF9bz75j5lGVdxy43hyAIgc82Eo3LVLz7Wg+6BOu5e72jTwEE6/Qpd9/u7gUEn9gHhWeuNAof00PuXhCG25Mc40Yypr5P3f18d2/q7oMJgmXWkSbh8K/L2LbewHJ3n+buJe6+DJhM0I0FwXGf0u+B2Pul27+0sT0SMzsPeAy4wN2zY5qWAylm1jVm2KFek18n+AD17iHmfR/B4xju7ruOUkoqwYkW1Y6CIBoZwHZ3329mgwiOIRyvR4B7LDzN08wyzeySw41sZgPCftymBLvsk8I9BQgC4RIza25mSWZ2NcGLe2VM+41mVjvsMx1D8CbGg37Y+cBPzKyWmX0dOJHD9LmWwSwg38x+GC4v2cx6m9nAcHl7CbpMbuXzDf8HBJ/4Y4PgWNf1ywTHQU4xszTgZ5TzfWLBKaK1wrtp4fo5uBGdDVwTPm+pBGf9bHT3reEn29UExzVSzKwBwR7dgiMsqxbBHgZA7HIxsxPMLMOCU1qvIjiI+6ewrZeZ9QvXcz3gj8AGgoPVmNmF4bEhC9fj7cA/wlnPA7pacAqpmVln4PyYOl8DGlpwqmeymV1McNzlv2H7kwTHFzqFwXs38EZM3Wnh4zAgNVx/SWHbWcDzwDfd/QuhFu7NvQr8zMzqWnB654UEB4xjXUvQffuFADKzewheL2cf3MOMaRty8DUSvj5/SLC3OvMQT0vVF3XfVE3749DHCH5RapyLCQ5u5hO84B8iPCODz/utU2LG/0LfMsEu/f/G3L+aoHvk4Jkx445Q3/vhcrcDjwJ1Y9pqEXRV5ITz+gg4L6a9I8EB3G3h9G8CXWPaO4SPfx/BwcGzY9quBBYdoa41lOqHJ+imGU9wDGAHMKPUPH8dLis9vH9buO6aH++6DodfS3AW0ZfOGiLod959jK+JNeFyYv86hG2NCTZkuQT95u8Dg2Km7Reu0x3AVoLuudjHtxs4NeZ+6eV4TNsdBGdg7QmXkxXTdlb4nO0Ja/l7qed2fLg+dgNLKXUKK8GpxwvD9byeoPsqKab9VILX6G6C00BPLTX9fWFtWwg21A1LvadKP64zwrZ3CI5J7I75mxozbaPwsewJn9MrSi23dTj9l86WCpdTUGrePwrbTic48HzwvfRv4LSotz/H+2fhgxIRkQSlriERkQSnIBARSXAKAhGRBKcgEBFJcAoCEZEEd8ifs61OmjRp4h06dIi6DBGRKmfu3Llb3b3p0car9kHQoUMH5syZE3UZIiJVjpl9Wpbx1DUkIpLgFAQiIglOQSAikuAUBCIiCU5BICKS4BQEIiIJTkEgIpLgFAQiIglOQSAikuCq/TeLRUTKy9354JNt7CssplvzDNo0rE1S0vFeRrz6URCISEJbvHEXP3tjETNWbf9sWO3UZLo0q0fX5vXo2iyDbs3r0a15Bq0b1MyAUBCISELatruAP05fzoRZa8msncrPL+xFz1aZrNicz/LNu1mRm89/V27l1Y82fDZNnbQwIJpl0LV5PbqFQVHdA0JBICIJpbCohGc+XMMD/1rBvsJirv1KB+4Y1o3MOqkADGjf8Avj5+07wMrcIByWb85nxebd/GfFFl75aP1n49RJS6Zrs3p0bZ7xWTh0bV6P1g1qY1b1A0JBICIJ452lufz8jcWs2rqH07s15f/O70mXZvWOOE1m7VQGtG/EgPaNvjA8b+8BlucGwbB8cz4rcvP59/ItvDz384Com5ZMl+YZdGsWdC11CbuYWmXWqlIBoSAQkRpvZW4+P39jCf9evoVOTevy5HUDObNHs3LNM7NOKgM7NGJghy8GxM69hZ/tPazMDf6/s2wLL8UERL30FLo0q/fZsYeuzTPo2qweLSMKCHP3Sl9oRcrKynJdj0BEDiVv7wHuf2s5z874lDppyXx3WFeuGdqBtJTKP3N+x55Clm/OZ3nublaEXUwrcvPZurvws3Ey0lOCvYbPjkEE/1vUP76AMLO57p511PHiFQRm9nvgAqAQ+AT4lrvvDNvuAW4AioHb3X1aOPw84AEgGXjc3X9ztOUoCESktKLiEsbPWsufpi8nb98BLh/Uju+f043G9dKjLu1LtocBEXuQesXm3WzbEwREo7ppfPR/5xzXvMsaBPHsGpoO3OPuRWb2W+Ae4Idm1hMYBfQCWgFvmVm3cJq/AucA64HZZjbJ3RfHsUYRqWHeX7GVn7+xmGWb8xnaqTE/vqAnJ7SsH3VZh9WobhpDOjVmSKfGXxi+bXcByzfvJm9f4WGmrDhxCwJ3/2fM3RnAxeHtC4EJ7l4ArDazlcCgsG2lu68CMLMJ4bgKAhE5qjVb9/DLKUuYvngz7RrV4ZGrBnBur+ZV6qDssWhcL52hlbQHU1kHi68HJoa3WxMEw0Hrw2EA60oNH3yomZnZGGAMQLt27Sq0UBGpXvL3H+Cht1cy7r+rSUtO4q7zunP9yR2plZocdWnVRrmCwMzeAlocouled/9HOM69QBHwfHmWFcvdxwJjIThGUFHzFZHqo7jEeXnuOn4/bRnb9hRycf823Hlud5rVrxV1adVOuYLA3c8+UruZXQecDwzzz49KbwDaxozWJhzGEYaLiHxm1urt3Pf6IhZt3EVW+4aMu24gJ7ZpEHVZ1VbcuobCM4DuAk53970xTZOAF8zsTwQHi7sCswADuppZR4IAGAVcEa/6RKT6Wb9jL7+eupTJC3JolVmLBy8/iQtObFltjwNUFfE8RvAQkA5MD5+kGe5+s7svMrMXCQ4CFwG3unsxgJndBkwjOH10nLsvimN9IlJN7C0s4uF3P2Hse6swg++d3Y0xp3WidpqOA1QEfaFMRKqskhLnHx9v4DdTl7J5VwEX9mvFD8/rQasGtaMurVqoCt8jEBE5bvPW7uC+1xczf91OTmyTyd+u7P+l3/uRiqEgEJEqZVPefn775lJem7eBphnp/OGSvnzjpNbV+meeqzoFgYhUCfsPFPPYe6v427ufUOzOLWd05pYzu1AvXZupeNMaFpFIuTtTsjfxqylL2LBzHyN6t+BHI0+gbaM6UZeWMBQEIhKZhRvy+Nnri5m1Zjs9WmQwfvQQhnZufPQJpUIpCESk0m3JL+AP05bx4tx1NKyTxq++3ofLBrYlWccBIqEgEJFKtX7HXkY+8B/2FhZzw8kd+c6wrmTWTo26rISmIBCRSvX3eRvYtb+IKbefSs9WVffnoRNJ5V+mR0QS2pTsTfRv10AhUIUoCESk0qzZuofFObsY2adl1KVIDAWBiFSaKQtzABihIKhSFAQiUmmmZOfQr20DWuu3gqoUBYGIVIq12/aycMMuRvY51LWsJEoKAhGpFJ91C/VWt1BVoyAQkUoxJTuHE9tk6qcjqiAFgYjE3brte1mwPk9nC1VRCgIRibupYbfQSHULVUkKAhGJuynZm+jduj7tGqtbqCpSEIhIXG3YuY/563aqW6gKUxCISFxNzVa3UFWnIBCRuJqSnUPPlvXp0KRu1KXIYSgIRCRuNu7cx0drd+pLZFWcgkBE4ubNhZsAdHygilMQiEjcTMnOoUeLDDo1rRd1KXIECgIRiYtNefuZ8+kO7Q1UAwoCEYmLNw9+iUxBUOUpCEQkLqYs3ES35vXo0kzdQlWdgkBEKlzurv3MXrNdewPVhIJARCrcm4s24a5uoepCQSAiFW5Kdg5dmtWjW/OMqEuRMlAQiEiF2pJfwKzV2xnZW18iqy4UBCJSoaYt2kSJw8gT1S1UXSgIRKRCTcnOoVPTunRXt1C1oSAQkQqzbXcBM1ZtY2TvlphZ1OVIGSkIRKTCTFu0OegW0tlC1YqCQEQqzJTsHDo0rsMJLdUtVJ0oCESkQmzfU8iHq7Yxso+6haobBYGIVIh/LtpEcYmrW6gaUhCISIWYsnAT7RrVoVer+lGXIsco7kFgZj8wMzezJuF9M7MHzWylmS0ws/4x415rZivCv2vjXZuIVIydewv5YOVWdQtVUynxnLmZtQWGA2tjBo8AuoZ/g4GHgcFm1gj4CZAFODDXzCa5+4541igi5ffPxZspKnFdkrKaivcewf3AXQQb9oMuBJ7xwAyggZm1BM4Fprv79nDjPx04L871iUgFmJKdQ5uGtenTOjPqUuQ4xC0IzOxCYIO7f1yqqTWwLub++nDY4YYfat5jzGyOmc3ZsmVLBVYtIscqb+8B/qtuoWqtXF1DZvYWcKh9wXuBHxF0C1U4dx8LjAXIysryo4wuInE0fclmDhTrbKHqrFxB4O5nH2q4mfUBOgIfh58Q2gAfmdkgYAPQNmb0NuGwDcAZpYa/W576RCT+pmTn0LpBbfq2UbdQdRWXriF3z3b3Zu7ewd07EHTz9Hf3TcAk4Jrw7KEhQJ675wDTgOFm1tDMGhLsTUyLR30iUjF27T/Af1ZsYUTvFuoWqsbietbQYUwBRgIrgb3AtwDcfbuZ/RyYHY73M3ffHkF9IlJGby0Ou4X0k9PVWqUEQbhXcPC2A7ceZrxxwLjKqElEym9K9iZaZtaiX5sGUZci5aBvFovIccnff4D3VmxhRO+WJCWpW6g6UxCIyHF5e2kuhUUl+hJZDaAgEJHjMnlBDs3rp9O/XcOoS5FyUhCIyDHbXVDEu8vVLVRTKAhE5Jh93i2ks4VqAgWBiByzKQtyaJqRzoD26haqCRQEInJM9hQU8c6yXEb0bkGyuoVqBAWBiByTd5blUqBuoRpFQSAix2Rq9iaa1EtnYIdGUZciFURBICJltq+wmLeX5nJe7+bqFqpBFAQiUmbvLMtl34FiRvZWt1BNoiAQkTKbkp1D47ppDOqobqGaREEgImWy/0DQLTS8VwtSkrXpqEn0bIpImby7bAt7C4v5qs4WqnEUBCJSJlOyc2hYJ5UhndQtVNMoCETkqPYfKOZfSzZzrrqFaiQ9oyJyVO8t38KewmJ9iayGUhCIyFFNXbiJBnVSGdq5cdSlSBwoCETkiAqKinlr8WaG92xOqrqFaiQ9qyJyRP9ZvpX8giJGqFuoxlIQiMgRTVmYQ/1aKZzcuUnUpUicKAhE5LAKioqZvngzw3u1IC1Fm4uaSs+siBzWByu3kb+/SBeor+EUBCJyWJOzc8iolcLJXdQtVJMpCETkkAqLSvjnok2cc0Jz0lOSoy5H4khBICKH9MEnW9m1v0hfIksACgIROaQp2TnUS0/hlK7qFqrpFAQi8iUHikv45+LNnH1CM2qlqluoplMQiMiXfPjJNnbuPaBuoQShIBCRL5m6MIe6acmc1q1p1KVIJVAQiMgXFBWXMG3RZoad0FzdQglCQSAiXzBz9Xa27ynUl8gSiIJARL5gcnYOddKSOaN7s6hLkUqiIBCRzxQVlzBt4SbO7KGzhRKJgkBEPjNrzXa27SnUBeoTjIJARD4zJTuHWqlJnNFdZwslEgWBiABQXOK8uXAzZ/VoRp20lKjLkUqkIBARAGav2c7W3QX6ElkCUhCICABTs3NIT0niTJ0tlHDiGgRm9h0zW2pmi8zsdzHD7zGzlWa2zMzOjRl+XjhspZndHc/aRORzJSXO1IWbOLN7M+qmq1so0cTtGTezM4ELgb7uXmBmzcLhPYFRQC+gFfCWmXULJ/srcA6wHphtZpPcfXG8ahSRwNy1O8jNL2CEvkSWkOIZ/d8GfuPuBQDunhsOvxCYEA5fbWYrgUFh20p3XwVgZhPCcRUEInE2eUEOaSlJDDuhedSlSATi2TXUDTjVzGaa2b/NbGA4vDWwLma89eGwww3/EjMbY2ZzzGzOli1b4lC6SOIIuoVyOL1bU+qpWyghletZN7O3gEPtS94bzrsRMAQYCLxoZp3Ks7yD3H0sMBYgKyvLK2KeIolq3rodbN5VoC+RJbByBYG7n324NjP7NvCquzswy8xKgCbABqBtzKhtwmEcYbiIxMnkBZtIS05i2Ak6WyhRxbNr6O/AmQDhweA0YCswCRhlZulm1hHoCswCZgNdzayjmaURHFCeFMf6RBLewW6h07o1IaNWatTlSETi2SE4DhhnZguBQuDacO9gkZm9SHAQuAi41d2LAczsNmAakAyMc/dFcaxPJOHNX7+TnLz93Hlu96hLkQjFLQjcvRC46jBtvwR+eYjhU4Ap8apJRL5oanYOqcmms4USnL5ZLJKg3J0p2Zs4tWtTMmurWyiRKQhEEtTH6/PYsHMfI3rrS2SJTkEgkqAOdgsN76kgSHQKApEE5O5Mzs7h5C5NyKyjbqFEpyAQSUALN+xi/Y59jOytL5GJgkAkIU3OziElyRjeS2cLiYJAJOG4B18iG9q5MQ3qpEVdjlQBCgKRBLNo4y4+3bZXvy0kn1EQiCSYKdk5JCcZw3vpbCEJKAhEEkjwJbIchnZqTKO66haSgIJAJIEsyclnzba9ukC9fIGCQCSBTF2YQ5Khs4XkCxQEIgni4JfIhnRqTJN66VGXI1WIgkAkQSzfvJtVW/YwQt1CUoqCQCRBTM7OwQzO09lCUoqCQCRBTM3OYVCHRjTNULeQfJGCQCQBrNicz4rc3Xz1RHULyZcpCEQSgLqF5EgUBCIJYGr2Jga2b0Sz+rWiLkWqIAWBSA23Mnc3yzbnM7KP9gbk0BQEIjXc1OwcAM7TtQfkMBQEIjXc5Owcsto3pEWmuoXk0BQEIjXYqi27WbopX18ikyNSEIjUUCUlzkNvrwRgRG8dH5DDUxCI1EDFJc5dryzg1Xkb+M5ZXWjVoHbUJUkVlhJ1ASJSsYqKS/iflz7m7/M38t1hXbnj7K5RlyRVnIJApAY5UFzC9ybO540FOfzP8G7cdpZCQI5OQSBSQxQWlXD7+Hm8uWgT94zowU2nd466JKkmFAQiNUBBUTG3Pv8Rby3J5cfn9+T6UzpGXZJUIwoCkWpu/4Fibnp2Lv9evoWfX9Sbq4e0j7okqWYUBCLV2L7CYkY/M4f/frKV336zD5cNbBd1SVINKQhEqqk9BUXc8PRsZq3ezh8u7ss3B7SJuiSpphQEItVQ/v4DfOvJ2cxbt5P7L+vHhf1aR12SVGMKApFqJm/fAa4dN4uFG/J4cNRJutiMlJuCQKQa2bm3kKufmMXSTbv465X9OVcXmpEKoCAQqSa27ynkysdn8knubh69egBn9WgedUlSQygIRKqBLfkFXPX4TNZs28Pj12ZxWremUZckNYiCQKSKy921n8sfm8HGnft58rqBfKVLk6hLkhpGQSBSheXk7eOKx2aSu2s/T18/iEEdG0VdktRAcfsZajPrZ2YzzGy+mc0xs0HhcDOzB81spZktMLP+MdNca2Yrwr9r41WbSHWwfsdeLnt0BlvzC3jmBoWAxE889wh+B9zn7lPNbGR4/wxgBNA1/BsMPAwMNrNGwE+ALMCBuWY2yd13xLFGkSpp7ba9XP7YDPL3H+DZGwfTr22DqEuSGiyeF6ZxoH54OxPYGN6+EHjGAzOABmbWEjgXmO7u28ON/3TgvDjWJ1Ilrd66h0sf/ZA9hUW8MHqIQkDiLp57BHcA08zsDwSB85VweGtgXcx468Nhhxv+JWY2BhgD0K6dfltFao6Vuflc8dhMikqc8aOHcELL+kefSKScyhUEZvYWcKhvtNwLDAO+5+6vmNmlwBPA2eVZ3kHuPhYYC5CVleUVMU+RqC3blM+Vj88AjAljhtCteUbUJUmCKFcQuPthN+xm9gzw3fDuS8Dj4e0NQNuYUduEwzYQHEOIHf5ueeoTqS4WbczjqsdnkpaSxAujh9C5ab2oS5IEEs9jBBuB08PbZwErwtuTgGvCs4eGAHnungNMA4abWUMzawgMD4eJ1GjZ6/O44rGZ1E5NZuKYoQoBqXTxPEYwGnjAzFKA/YR9+sAUYCSwEtgLfAvA3beb2c+B2eF4P3P37XGsTyRy89bu4Jpxs8isncr40UNo26hO1CVJAopbELj7+8CAQwx34NbDTDMOGBevmkSqkjlrtnPdk7NpXC+NF0YPoXWD2lGXJAkqnl1DInIYH36yjWvGzaJZRjoTxwxVCEikFAQilez9FVv51lOzaN2gNhNuGkKLzFpRlyQJTr81JFKJ3l2Wy5hn59KpSV2eu3EwTeqlR12SiIJApLK8tXgztzz/EV2b1+O5GwbTsG5a1CWJAOoaEqkUby7M4ebn5nJCywxeuHGIQkCqFO0RiMTZ6x9v5I6J8+nbJpOnrh9E/VqpUZdfop2xAAAVnklEQVQk8gUKApE4em3een7w4sdktW/EuG8NpF663nJS9ehVKRInL81Zx12vLGBIx8Y8cV0WddL0dpOqSccIROLghZlrufPlBZzSpQnjrhuoEJAqTa9OkQr2zIdr+PE/FnFm96Y8fNUAaqUmR12SyBEpCEQq0OP/WcUvJi/hnJ7NeeiKk0hPUQhI1acgEKkgB0NgZJ8WPDDqJFKT1fMq1YOCQKQCzFy1jV9NCULgwVEnkaIQkGpEr1aRctq5t5A7Js6nfeO6/P7ivgoBqXa0RyBSDu7O3a9ks3V3Aa9++2Tq6nsCUg3po4tIOUyYvY43F23iznO706dNZtTliBwXBYHIcVqZm899ry/i1K5NuPGUTlGXI3LcFAQix6GgqJjvjJ9PnbQU/nhJX5KSLOqSRI6bOjRFjsNvpy5jSc4uxl2XRbP6urCMVG/aIxA5Ru8sy2Xcf1dz3Vc6cFaP5lGXI1JuCgKRY7Alv4A7X/qYHi0yuHtEj6jLEakQ6hoSKaOSEucHL31M/v4ixo8eot8QkhpDewQiZTTuv6t5b/kW/u/8nnRtnhF1OSIVRkEgUgYLN+Tx2zeXMrxnc64c3C7qckQqlIJA5Cj2FhZx+4R5NK6bzm+/eSJmOlVUahYdIxA5ip+9vpjVW/fw/I2DddF5qZG0RyBH5e5RlxCZKdk5TJi9jlvO6MxXOjeJuhyRuNAegRzSvsJiJmfnMHH2WpZtymfsNVkM6dQ46rIq1Yad+7j7lQX0a9uAO87uFnU5InGjIJAvWLghj/Gz1jJp/kbyC4ro2KQujeqmcd2Ts3jsmixO7do06hIrRXGJc8eEeZQ4PKiLzEgNpyAQ8vYdYNL8DUyYvY5FG3eRnpLEV/u05LKBbRnUsRHb9hRy1eMzueHpOTxyVf+E+DbtQ2+vZPaaHfz5sn60a1wn6nJE4kpBkKDcnVmrtzNx9jomZ+dQUFTCCS3r87MLe3Fh39Zk1kn9bNwm9dIZP3oI14ybxU3PzuUvl/fnvN4tIqw+vuas2c4D/1rO109qzUUntY66HJG4s+p+IDArK8vnzJkTdRnVxpb8Al75aD0vzl7Hqq17yEhP4Wv9WjFqYDt6t65/xFMj8/Yd4LonZ7FgfR73X9aPr/VtVYmVV468fQcY+cB/SE4yJt9+Chm1Uo8+kUgVZWZz3T3raONpjyABFJc47y3fwoTZa/nXklyKSpyBHRpyy5ldGNmnBXXSyvYyyKydyrM3DOb6J2dzx4R5FBaVcPGANnGuvvK4O/e+ls3mXft56eahCgFJGAqCGmzd9r28NGcdL81dT07efhrXTeP6UzpyaVZbujSrd1zzrJeewlPXD2T0M3O48+WPKSwq4Yoa8k3bl+au540FOdx5bndOatcw6nJEKo2CoIYpKCrmrcW5TJi9lvdXbgXgtK5N+fH5PRl2QnPSUsp/9kudtBSeuHYg335uLj96LZvComKuO7ljuecbpVVbdvPTSYsY2qkxN5/eOepyRCqVgqCGWLE5n4mz1/HqvA1s31NIq8xafHdYVy7JakvrBrUrfHm1UpN55OoBfOeFefz09cUUFpcw5rTquQEtLCrhuxPmk5aSxP2X9SNZVxuTBKMgqMb2FhbxxoIcJs5ex9xPd5CSZAzv1ZzLBrbjlC5N4r5BS09J5q9X9ueOifP51ZSlFBwo4TvDusZ1mfHwh38uI3tDHmOvHkCLTF1tTBKPgqCacXcWrM9jwux1vP7xRnYXFNGpaV1+NLIH3+jfhib10iu1ntTkJB64rB/pyUn8cfpyCopK+MHwbtXmh9n+s2ILY99bxVVD2jG8V809JVbkSBQE1UTe3gP8PfzS15KcXdRKTeKrfVoxalBbsto3jHTDm5KcxO8v6UtaShIPvbOSwuIS7hnRo8qHwbbdBXz/xY/p2qwe947sGXU5IpEpVxCY2SXAT4ETgEHuPiem7R7gBqAYuN3dp4XDzwMeAJKBx939N+HwjsAEoDEwF7ja3QvLU1915+7MWLWdibPXMmXhJgqLSujTOpNfXNSbr/VrRf0qdHpjcpLxq6/3IS0libHvraLgQDE/uaAXSVW0v93dufPlBeTtO8CzNwyidpquNiaJq7x7BAuBbwCPxg40s57AKKAX0Ap4y8wO/mrXX4FzgPXAbDOb5O6Lgd8C97v7BDN7hCBEHi5nfdVS7q79vBx+6WvNtr1k1Eph1MC2XJrVlt6tM6Mu77CSkoz7vtaLtOQkHn9/NYXFJfzyoj5VMgye/mANby/N5b6v9aJHi/pRlyMSqXIFgbsvAQ7VBXAhMMHdC4DVZrYSGBS2rXT3VeF0E4ALzWwJcBZwRTjO0wR7GgkVBAvW7+Qvb6/k7aW5FJc4gzo24vZhXRnRu2W1+cRqZtz71RNIT03ir+98QkFRCb/75omkVKEfbVuSs4tfTV3KsB7NuGZo+6jLEYlcvI4RtAZmxNxfHw4DWFdq+GCC7qCd7l50iPG/xMzGAGMA2rWrGV9mWrVlN1c+PpO05CRuPLUjl2W1pVPT4/vSV9TMjDvP7UGtlGT+OH05hUUl3H9ZvyrxC577Cou5ffw8Mmun8ruLdbUxEShDEJjZW8ChTqe4193/UfElHZ27jwXGQvBbQ1HUUJHy9x9g9DNzSE1O4h+3nUybhjXj1y6/M6wraSlJ/HrqUgqLSvjLFSeRnhLtns0vJi9mRe5unr1hEI0r+QwrkarqqEHg7mcfx3w3AG1j7rcJh3GY4duABmaWEu4VxI5fo5WUON+bOJ812/by3A2Da0wIHHTT6Z1JT0nip68v5uZn5/LwVQOolRpNGLy5cBPPz1zLTad1SpjrKoiURbz21ScBo8wsPTwbqCswC5gNdDWzjmaWRnBAeZIHP4H6DnBxOP21QCR7G5Xt/reW89aSXH58fk+Gdq6ZVwC77uSO/OrrfXh3+RZufHoO+wqLK72GnLx93P3qAvq0zuQHw7tX+vJFqrJyBYGZfd3M1gNDgclmNg3A3RcBLwKLgTeBW929OPy0fxswDVgCvBiOC/BD4PvhgeXGwBPlqa06mJqdw1/eXsmlWW1q/EHLKwa34/cX9+WDT7Zy7ZOz2F1QdPSJKkhxuNdVWFTCg5efVCG/tyRSk+h6BBFZumkX3/jbB3RvkcGEMUMi7zuvLP+Yv4Hvv/gxfdtk8tT1gyrluxB/fWclv5+2jN9ffCKXZLU9+gQiNURZr0egj0YR2Lm3kNHPzKFeegqPXDUgYUIA4MJ+rfnrFSeRvSGPqx6fyc698f3O4Ly1O/jT9OVc0LdVjbp2gkhFUhBUsqLiEm57YR6b8wp45OoBNK+feD9ydl7vljxy1QCW5uRz+WMz2ba7IC7Lyd9/gNsnzKNF/Vr84qLeOlVU5DAUBJXsN1OX8v7Krfziot70T+CLnww7oTmPX5vF6q27GTV2Brn5+yt8Gf/394Vs3LmfBy/vR2btqvNzHCJVjYKgEr360Xoef3811w5tz6UD1Vd9WremPHndIDbs3MeoR2eQk7evwub92rz1/H3+Rr47rCsD2jeqsPmK1EQKgkqyYP1O7n41myGdGvG/5+uXLg8a2rkxz94wiC35BVz66Ies27633PP8dNse/ve1hQzq0Ihbz+xSAVWK1GwKgkqwJb+Am56dS9N66fz1iv5V4qcWqpIB7Rvx3I2Dydt7gMse/ZA1W/cc97wOFJdw+4T5JCcZ94/S1cZEykJbpDgrLCrh28/NZcfeQsZeM0A/a3AYfds2YPyYIew7UMylj37IytzdxzWf+6cv5+N1O/nNN0+MyyU6RWoiBUGc/fT1Rcz5dAe/v7gvvVpV3Z+Qrgp6tcpkwpihlDiMGvshSzftOqbpP1i5lYf//QmjBrZlZJ+WcapSpOZREMTR8zM/5YWZa7n59M5c0LdV1OVUC91bZDDxpiEkJxmjxs5g4Ya8Mk23Y08h33txPh2b1OXHF+gYjMixUBDEyew12/nJPxZxRvem3HmuftvmWHRuWo8XbxpK3bQUrnhsBvPW7jji+O7OXa8sYMeeAzw46iTqpOkKrCLHQkEQBxt37uPbz82lbaM6PDDqJB2wPA7tG9dl4k1DaFg3jaufmMXsNdsPO+5zM9cyffFm7jqve5W+gptIVaUgqGD7DxRz07Nz2X+ghMeuGaAvMpVDm4Z1mDhmKM3qp3PNE7P4YOXWL42zfHM+v3hjMad3a8r1J3eMoEqR6k9BUIHcnR+9mk32hjzuv6wfXZplRF1StdcisxYTxwylXaM6fOup2fx7+ZbP2vYfCK42llErhT9c0rdKXhtZpDpQEFSgJ95fzavzNvD9c7pxTs/mUZdTYzTNSGf8mCF0blqP0U/PYfrizQD8esoSlm7K5w+X9KVphk7LFTleCoIK8v6KrfxqyhLO69WC2/Rt1grXqG4a40cP4YSWGXz7ubnc9/oinv7wU244pSNndG8WdXki1ZqCoAKs3baX28Z/RNdmGfzxUnVRxEtmnVSeu3Ew/do24Mn/rqFny/rcdZ7OyBIpL51nV057CooY/cwc3GHsNQOom65VGk8ZtVJ5+vpBPPreKi7u3yahruUgEi/aapWDu/M/L33Mitx8nr5+EO0b1426pIRQNz2F75/TLeoyRGoMdQ2Vw0Nvr2Tqwk3cM+IETu3aNOpyRESOi4LgOL21eDN/nL6cr5/UmhtP1fnrIlJ9KQiOw8rcfO6YOJ8+rTP59Tf66BKIIlKtKQiOUd6+A4x+Zi61UpN49OoB1ErVwUoRqd50sPgYFJc4350wj3Xb9/LC6CG00u/di0gNoCA4Bn/85zLeXbaFX1zUm0EddR1cEakZ1DVURm8s2Mjf3v2Eywe146oh7aMuR0SkwigIymDxxl3c+dICsto35L6v9Yq6HBGRCqUgOIrtewoZ/cwcMmun8rer+pOWolUmIjWLjhEcwYHiEm59/iO27C7g5ZuH0iyjVtQliYhUOH28PYJfTl7Ch6u28Ztv9OHENg2iLkdEJC4UBIfx4px1PPXBGm44pSPf6N8m6nJEROJGQXAI89bu4H9fW8gpXZpwz4geUZcjIhJXCoJScnft5+bn5tI8M52/XH4SKclaRSJSs+lgcYyComJuem4u+fuLePWWr9CwblrUJYmIxJ2CIOTu/Pjvi5i3dicPX9mfHi3qR12SiEilUL9H6LkZnzJxzjpuO7MLI/q0jLocEZFKoyAAZq7axn2vL2ZYj2a68pWIJJyED4INO/dxy/Mf0a5xHe4f1U8XnheRhJPQQbCvsJgxz8yhsKiEx67Jon6t1KhLEhGpdAl7sNjd+eErC1ics4tx1w6kc9N6UZckIhKJcu0RmNklZrbIzErMLCtm+DlmNtfMssP/Z8W0DQiHrzSzBy28zqOZNTKz6Wa2IvzfsDy1Hc3Y91Yx6eON/M/w7pzZo1k8FyUiUqWVt2toIfAN4L1Sw7cCF7h7H+Ba4NmYtoeB0UDX8O+8cPjdwL/cvSvwr/B+XGzetZ8/Tl/OV09syS1ndI7XYkREqoVydQ25+xLgSxdvd/d5MXcXAbXNLB1oBNR39xnhdM8AFwFTgQuBM8JpngbeBX5YnvoOp3n9WkwcM4TuLTJ04XkRSXiVcbD4m8BH7l4AtAbWx7StD4cBNHf3nPD2JqD54WZoZmPMbI6ZzdmyZctxFXVSu4bUSUvYQyQiIp856pbQzN4CWhyi6V53/8dRpu0F/BYYfixFububmR+hfSwwFiArK+uw44mIyNEdNQjc/ezjmbGZtQFeA65x90/CwRuA2N90bhMOA9hsZi3dPcfMWgK5x7NcERE5NnHpGjKzBsBk4G53/+/B4WHXzy4zGxKeLXQNcHCvYhLBgWXC/0fc2xARkYpR3tNHv25m64GhwGQzmxY23QZ0AX5sZvPDv4PnaN4CPA6sBD4hOFAM8BvgHDNbAZwd3hcRkTgz9+rdxZ6VleVz5syJugwRkSrHzOa6e9bRxkvon5gQEREFgYhIwlMQiIgkOAWBiEiCUxCIiCS4an/WkJltAT49zsmbEPxAXlWimsquKtalmsquKtZV02pq7+5NjzZStQ+C8jCzOWU5taoyqaayq4p1qaayq4p1JWpN6hoSEUlwCgIRkQSX6EEwNuoCDkE1lV1VrEs1lV1VrCsha0roYwQiIqI9AhGRhJeQQWBm55nZMjNbaWZxuzbysTCzcWaWa2YLo67lIDNra2bvmNliM1tkZt+tAjXVMrNZZvZxWNN9Udd0kJklm9k8M3sj6loOMrM1ZpYd/gJwlfh1RjNrYGYvm9lSM1tiZkMjrqd7zK8kzzezXWZ2R5Q1HWRm3wtf5wvNbLyZ1YrLchKta8jMkoHlwDkEl8qcDVzu7osjrus0YDfwjLv3jrKWg8ILBLV094/MLAOYC1wU5boKr2NR1913m1kq8D7w3YPXwY6SmX0fyCK4Lvf5UdcDQRAAWe5eZc6NN7Ongf+4++NmlgbUcfedUdcFn20fNgCD3f14v59UUbW0Jnh993T3fWb2IjDF3Z+q6GUl4h7BIGClu69y90JgAnBhxDXh7u8B26OuI5a757j7R+HtfGAJn19jOqqa3N13h3dTw7/IP82EV+T7KsG1NuQwzCwTOA14AsDdC6tKCISGAZ9EHQIxUoDaZpYC1AE2xmMhiRgErYF1MffXE/HGrTowsw7AScDMaCv5rAtmPsHlTKe7e+Q1AX8G7gJKoi6kFAf+aWZzzWxM1MUAHYEtwJNhN9rjZlY36qJijALGR10EgLtvAP4ArAVygDx3/2c8lpWIQSDHyMzqAa8Ad7j7rqjrcfdid+9HcM3rQWYWaVeamZ0P5Lr73CjrOIxT3L0/MAK4NeyCjFIK0B942N1PAvYAVeU4XRrwNeClqGsBMLOGBL0VHYFWQF0zuyoey0rEINgAtI253yYcJocQ9sO/Ajzv7q9GXU+ssEvhHeC8iEs5Gfha2B8/ATjLzJ6LtqRA+KkSd88FXiPoGo3SemB9zF7cywTBUBWMAD5y981RFxI6G1jt7lvc/QDwKvCVeCwoEYNgNtDVzDqGnwBGAZMirqlKCg/MPgEscfc/RV0PgJk1NbMG4e3aBAf9l0ZZk7vf4+5t3L0DwevpbXePyye3Y2FmdcOD/ITdL8OBSM9Kc/dNwDoz6x4OGgZEeqJGjMupIt1CobXAEDOrE74XhxEcp6twKfGYaVXm7kVmdhswDUgGxrn7oojLwszGA2cATcxsPfATd38i2qo4GbgayA775AF+5O5TIqypJfB0eHZHEvCiu1eZ0zWrmObAa8E2hBTgBXd/M9qSAPgO8Hz4QWwV8K2I6zkYlOcAN0Vdy0HuPtPMXgY+AoqAecTpW8YJd/qoiIh8USJ2DYmISAwFgYhIglMQiIgkOAWBiEiCUxCIiCQ4BYGISIJTEIiIJDgFgYhIgvt//nuSY9dkQ+4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "state = envs.reset()\n",
    "early_stop = False\n",
    "\n",
    "while frame_idx < max_frames and not early_stop:\n",
    "\n",
    "    log_probs = []\n",
    "    values    = []\n",
    "    states    = []\n",
    "    actions   = []\n",
    "    rewards   = []\n",
    "    masks     = []\n",
    "    entropy = 0\n",
    "\n",
    "    for _ in range(num_steps):\n",
    "        state = torch.FloatTensor(state).to(device)\n",
    "        dist, value = model(state)\n",
    "\n",
    "        action = dist.sample()\n",
    "        next_state, reward, done, _ = envs.step(action.cpu().numpy())\n",
    "\n",
    "        log_prob = dist.log_prob(action)\n",
    "        entropy += dist.entropy().mean()\n",
    "        \n",
    "        log_probs.append(log_prob)\n",
    "        values.append(value)\n",
    "        rewards.append(torch.FloatTensor(reward).unsqueeze(1).to(device))\n",
    "        masks.append(torch.FloatTensor(1 - done).unsqueeze(1).to(device))\n",
    "        \n",
    "        states.append(state)\n",
    "        actions.append(action)\n",
    "        \n",
    "        state = next_state\n",
    "        frame_idx += 1\n",
    "        \n",
    "        if frame_idx % 1000 == 0:\n",
    "            test_reward = np.mean([test_env() for _ in range(10)])\n",
    "            test_rewards.append(test_reward)\n",
    "            plot(frame_idx, test_rewards)\n",
    "            if test_reward > threshold_reward: early_stop = True\n",
    "            \n",
    "\n",
    "    next_state = torch.FloatTensor(next_state).to(device)\n",
    "    _, next_value = model(next_state)\n",
    "    returns = compute_gae(next_value, rewards, masks, values)\n",
    "\n",
    "    returns   = torch.cat(returns).detach()\n",
    "    log_probs = torch.cat(log_probs).detach()\n",
    "    values    = torch.cat(values).detach()\n",
    "    states    = torch.cat(states)\n",
    "    actions   = torch.cat(actions)\n",
    "    advantage = returns - values\n",
    "    \n",
    "    ppo_update(ppo_epochs, mini_batch_size, states, actions, log_probs, returns, advantage)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Saving trajectories for GAIL</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode: 0 reward: -133.5056485070341\n",
      "episode: 1 reward: -3.3737309166625002\n",
      "episode: 2 reward: -135.0328820133956\n",
      "episode: 3 reward: -131.27964142064513\n",
      "episode: 4 reward: -125.12845453838382\n",
      "episode: 5 reward: -4.247933460422459\n",
      "episode: 6 reward: -395.59297834503883\n",
      "episode: 7 reward: -253.25736991568547\n",
      "episode: 8 reward: -135.50603026103278\n",
      "episode: 9 reward: -132.72095459732952\n",
      "episode: 10 reward: -133.89608385869212\n",
      "episode: 11 reward: -4.5990508813314035\n",
      "episode: 12 reward: -134.44470210766775\n",
      "episode: 13 reward: -801.7661346371387\n",
      "episode: 14 reward: -131.97725229377644\n",
      "episode: 15 reward: -266.76940521674015\n",
      "episode: 16 reward: -247.5062278004002\n",
      "episode: 17 reward: -4.914595620774103\n",
      "episode: 18 reward: -138.7990887577753\n",
      "episode: 19 reward: -268.3754189751262\n",
      "episode: 20 reward: -363.28764882256417\n",
      "episode: 21 reward: -128.15870842354997\n",
      "episode: 22 reward: -134.94598918501788\n",
      "episode: 23 reward: -309.9577786212293\n",
      "episode: 24 reward: -131.91670030817002\n",
      "episode: 25 reward: -134.65823444568952\n",
      "episode: 26 reward: -134.5615349098279\n",
      "episode: 27 reward: -273.5740578550409\n",
      "episode: 28 reward: -265.05553942459926\n",
      "episode: 29 reward: -258.0591054576666\n",
      "episode: 30 reward: -128.91060595426686\n",
      "episode: 31 reward: -656.2461074160591\n",
      "episode: 32 reward: -136.84071690580248\n",
      "episode: 33 reward: -259.2365200533221\n",
      "episode: 34 reward: -132.68644155022494\n",
      "episode: 35 reward: -260.66364797902054\n",
      "episode: 36 reward: -128.8211009270027\n",
      "episode: 37 reward: -384.53615237759317\n",
      "episode: 38 reward: -4.612904346743044\n",
      "episode: 39 reward: -401.1162060114804\n",
      "episode: 40 reward: -126.25334578262932\n",
      "episode: 41 reward: -3.845934927726255\n",
      "episode: 42 reward: -132.44253012402612\n",
      "episode: 43 reward: -134.1267203432647\n",
      "episode: 44 reward: -128.56866661753938\n",
      "episode: 45 reward: -4.97856955649956\n",
      "episode: 46 reward: -392.498679426522\n",
      "episode: 47 reward: -4.756869243844947\n",
      "episode: 48 reward: -4.59189846851519\n",
      "episode: 49 reward: -4.7496626929539225\n",
      "episode: 50 reward: -131.08999767991665\n",
      "episode: 51 reward: -138.17235302513578\n",
      "episode: 52 reward: -3.751761058079555\n",
      "episode: 53 reward: -260.6317126814632\n",
      "episode: 54 reward: -4.535299319594524\n",
      "episode: 55 reward: -133.70892423024802\n",
      "episode: 56 reward: -134.8732103854694\n",
      "episode: 57 reward: -5.315182694344295\n",
      "episode: 58 reward: -265.04898120165\n",
      "episode: 59 reward: -124.99288470795233\n",
      "episode: 60 reward: -4.247632479535832\n",
      "episode: 61 reward: -3.68334723705883\n",
      "episode: 62 reward: -133.617727327027\n",
      "episode: 63 reward: -136.28353948776376\n",
      "episode: 64 reward: -5.056124136459314\n",
      "episode: 65 reward: -262.7844771770983\n",
      "episode: 66 reward: -251.52420165781922\n",
      "episode: 67 reward: -133.4014820950796\n",
      "episode: 68 reward: -7.0558924646711\n",
      "episode: 69 reward: -135.41150554590206\n",
      "episode: 70 reward: -131.8871841825757\n",
      "episode: 71 reward: -130.8724972571845\n",
      "episode: 72 reward: -367.7339135957503\n",
      "episode: 73 reward: -134.25198778254116\n",
      "episode: 74 reward: -133.86858295338342\n",
      "episode: 75 reward: -378.9443227440811\n",
      "episode: 76 reward: -3.5473336732949625\n",
      "episode: 77 reward: -261.5470895641183\n",
      "episode: 78 reward: -408.34135925288217\n",
      "episode: 79 reward: -257.6727990499033\n",
      "episode: 80 reward: -399.78682205537433\n",
      "episode: 81 reward: -266.08087229456055\n",
      "episode: 82 reward: -817.186490578741\n",
      "episode: 83 reward: -4.500140134501902\n",
      "episode: 84 reward: -508.65456581456573\n",
      "episode: 85 reward: -378.46002005145874\n",
      "episode: 86 reward: -137.76181809972095\n",
      "episode: 87 reward: -674.8280917415572\n",
      "episode: 88 reward: -128.65034230393303\n",
      "episode: 89 reward: -3.922315525193146\n",
      "episode: 90 reward: -131.00005239353024\n",
      "episode: 91 reward: -130.68974732718007\n",
      "episode: 92 reward: -135.21946982972375\n",
      "episode: 93 reward: -137.3667851983452\n",
      "episode: 94 reward: -136.9119001250973\n",
      "episode: 95 reward: -254.5371556381929\n",
      "episode: 96 reward: -374.827391591992\n",
      "episode: 97 reward: -523.9964989484117\n",
      "episode: 98 reward: -133.94200200894622\n",
      "episode: 99 reward: -133.74880434577523\n",
      "episode: 100 reward: -247.32247835568552\n",
      "episode: 101 reward: -138.75528548988993\n",
      "episode: 102 reward: -4.847096453940289\n",
      "episode: 103 reward: -136.62732481247133\n",
      "episode: 104 reward: -262.20300946977864\n",
      "episode: 105 reward: -6.5435854338994\n",
      "episode: 106 reward: -125.17361036750681\n",
      "episode: 107 reward: -690.5202921080676\n",
      "episode: 108 reward: -280.53617631459497\n",
      "episode: 109 reward: -135.40352441695322\n",
      "episode: 110 reward: -131.07617970631023\n",
      "episode: 111 reward: -247.0260554601557\n",
      "episode: 112 reward: -135.40673404514774\n",
      "episode: 113 reward: -395.03306256658476\n",
      "episode: 114 reward: -384.1784417792837\n",
      "episode: 115 reward: -128.4500742980931\n",
      "episode: 116 reward: -463.6977661877445\n",
      "episode: 117 reward: -130.94801971085445\n",
      "episode: 118 reward: -144.0228791279258\n",
      "episode: 119 reward: -667.2634492717342\n",
      "episode: 120 reward: -131.79948959004724\n",
      "episode: 121 reward: -138.03140142705894\n",
      "episode: 122 reward: -129.26779443720966\n",
      "episode: 123 reward: -3.2877798185337896\n",
      "episode: 124 reward: -134.72016283865193\n",
      "episode: 125 reward: -382.2159098741087\n",
      "episode: 126 reward: -264.6491917411121\n",
      "episode: 127 reward: -134.2254720027939\n",
      "episode: 128 reward: -424.8235005744391\n",
      "episode: 129 reward: -134.52619102883028\n",
      "episode: 130 reward: -537.7406839640856\n",
      "episode: 131 reward: -133.90654715605245\n",
      "episode: 132 reward: -132.20198118805123\n",
      "episode: 133 reward: -400.3589991495165\n",
      "episode: 134 reward: -130.12695949420717\n",
      "episode: 135 reward: -290.86810229081595\n",
      "episode: 136 reward: -394.9043391522139\n",
      "episode: 137 reward: -133.42125091255778\n",
      "episode: 138 reward: -134.96306459417266\n",
      "episode: 139 reward: -3.8499366797706336\n",
      "episode: 140 reward: -3.828788719469504\n",
      "episode: 141 reward: -5.554963437941836\n",
      "episode: 142 reward: -4.510403163975261\n",
      "episode: 143 reward: -325.97799775791754\n",
      "episode: 144 reward: -3.1174779530363375\n",
      "episode: 145 reward: -134.55262416681552\n",
      "episode: 146 reward: -350.45777263184095\n",
      "episode: 147 reward: -137.33235583532627\n",
      "episode: 148 reward: -452.0061280718382\n",
      "episode: 149 reward: -265.98673902850385\n",
      "episode: 150 reward: -284.8590382363739\n",
      "episode: 151 reward: -250.06981206461143\n",
      "episode: 152 reward: -129.50428228187013\n",
      "episode: 153 reward: -393.09302439930724\n",
      "episode: 154 reward: -5.075964808667517\n",
      "episode: 155 reward: -129.83816358490287\n",
      "episode: 156 reward: -266.1020126434327\n",
      "episode: 157 reward: -132.23463644630868\n",
      "episode: 158 reward: -779.5855091317233\n",
      "episode: 159 reward: -3.763971510946643\n",
      "episode: 160 reward: -132.67794144748086\n",
      "episode: 161 reward: -662.5587064643477\n",
      "episode: 162 reward: -135.2401324340408\n",
      "episode: 163 reward: -259.9633585943629\n",
      "episode: 164 reward: -6.232862086437321\n",
      "episode: 165 reward: -139.498411973157\n",
      "episode: 166 reward: -135.35070491390638\n",
      "episode: 167 reward: -135.1400077480551\n",
      "episode: 168 reward: -347.3664683729514\n",
      "episode: 169 reward: -427.1984854733556\n",
      "episode: 170 reward: -5.15672209428849\n",
      "episode: 171 reward: -525.916662268042\n",
      "episode: 172 reward: -133.7053511504196\n",
      "episode: 173 reward: -271.26784680564384\n",
      "episode: 174 reward: -124.85474506625023\n",
      "episode: 175 reward: -134.19873581079943\n",
      "episode: 176 reward: -255.83160338962983\n",
      "episode: 177 reward: -135.13400569542506\n",
      "episode: 178 reward: -4.960226836538054\n",
      "episode: 179 reward: -139.19809065222032\n",
      "episode: 180 reward: -140.05080094044732\n",
      "episode: 181 reward: -137.76647105767526\n",
      "episode: 182 reward: -403.1731636539886\n",
      "episode: 183 reward: -257.970427512537\n",
      "episode: 184 reward: -3.7473226459331066\n",
      "episode: 185 reward: -278.3098063643893\n",
      "episode: 186 reward: -255.99692458401518\n",
      "episode: 187 reward: -4.6365121508813445\n",
      "episode: 188 reward: -244.67627722290948\n",
      "episode: 189 reward: -131.21920785362062\n",
      "episode: 190 reward: -777.3698354491825\n",
      "episode: 191 reward: -132.07220706141683\n",
      "episode: 192 reward: -392.09434598281683\n",
      "episode: 193 reward: -136.06354238422503\n",
      "episode: 194 reward: -377.4409927865957\n",
      "episode: 195 reward: -132.18253486880235\n",
      "episode: 196 reward: -129.15162595976702\n",
      "episode: 197 reward: -396.5254064840202\n",
      "episode: 198 reward: -3.610361833207753\n",
      "episode: 199 reward: -245.53736015092704\n",
      "episode: 200 reward: -270.99181854480565\n",
      "episode: 201 reward: -247.4231450110685\n",
      "episode: 202 reward: -131.59894474370887\n",
      "episode: 203 reward: -144.7898370619998\n",
      "episode: 204 reward: -926.5588068852352\n",
      "episode: 205 reward: -133.39727923189105\n",
      "episode: 206 reward: -131.93566436017008\n",
      "episode: 207 reward: -6.40529176710689\n",
      "episode: 208 reward: -257.08448208556194\n",
      "episode: 209 reward: -130.92098423630432\n",
      "episode: 210 reward: -262.2927047192545\n",
      "episode: 211 reward: -6.859901180492491\n",
      "episode: 212 reward: -262.70877767928914\n",
      "episode: 213 reward: -134.56588203218894\n",
      "episode: 214 reward: -135.22465193371625\n",
      "episode: 215 reward: -137.9657247788344\n",
      "episode: 216 reward: -135.13425433384725\n",
      "episode: 217 reward: -132.3215993693809\n",
      "episode: 218 reward: -400.611961792729\n",
      "episode: 219 reward: -401.91908212383294\n",
      "episode: 220 reward: -282.5082305011229\n",
      "episode: 221 reward: -135.42191465289923\n",
      "episode: 222 reward: -399.7881535647735\n",
      "episode: 223 reward: -131.06522770318847\n",
      "episode: 224 reward: -130.7681491912167\n",
      "episode: 225 reward: -135.31477016876133\n",
      "episode: 226 reward: -3.914901001828447\n",
      "episode: 227 reward: -134.5129393394648\n",
      "episode: 228 reward: -376.1469783238271\n",
      "episode: 229 reward: -133.09045533066046\n",
      "episode: 230 reward: -383.2750315233141\n",
      "episode: 231 reward: -263.71240275232276\n",
      "episode: 232 reward: -500.0083919266878\n",
      "episode: 233 reward: -135.22531187168758\n",
      "episode: 234 reward: -135.17818433537522\n",
      "episode: 235 reward: -395.9834332194123\n",
      "episode: 236 reward: -126.08778928679216\n",
      "episode: 237 reward: -413.7495701300203\n",
      "episode: 238 reward: -131.37116502717876\n",
      "episode: 239 reward: -121.6506938627967\n",
      "episode: 240 reward: -653.7053929625495\n",
      "episode: 241 reward: -254.87183145095838\n",
      "episode: 242 reward: -129.71331746419523\n",
      "episode: 243 reward: -265.9795936355916\n",
      "episode: 244 reward: -400.65989274385277\n",
      "episode: 245 reward: -251.82522565834446\n",
      "episode: 246 reward: -3.95924871368981\n",
      "episode: 247 reward: -312.7505665224348\n",
      "episode: 248 reward: -135.5875093701436\n",
      "episode: 249 reward: -441.6053043293015\n"
     ]
    }
   ],
   "source": [
    "from itertools import count\n",
    "\n",
    "max_expert_num = 50000\n",
    "num_steps = 0\n",
    "expert_traj = []\n",
    "\n",
    "for i_episode in count():\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "    \n",
    "    while not done:\n",
    "        state = torch.FloatTensor(state).unsqueeze(0).to(device)\n",
    "        dist, _ = model(state)\n",
    "        action = dist.sample().cpu().numpy()[0]\n",
    "        next_state, reward, done, _ = env.step(action)\n",
    "        state = next_state\n",
    "        total_reward += reward\n",
    "        expert_traj.append(np.hstack([state, action]))\n",
    "        num_steps += 1\n",
    "    \n",
    "    print(\"episode:\", i_episode, \"reward:\", total_reward)\n",
    "    \n",
    "    if num_steps >= max_expert_num:\n",
    "        break\n",
    "        \n",
    "expert_traj = np.stack(expert_traj)\n",
    "print()\n",
    "print(expert_traj.shape)\n",
    "print()\n",
    "np.save(\"expert_traj.npy\", expert_traj)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch4]",
   "language": "python",
   "name": "conda-env-pytorch4-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
